import json
from time import sleep
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import pandas as pd

from lxml import etree, html
data = []


def main():
    browser = webdriver.Chrome()
    browser.get('http://www.taobao.com')
    add_cookies(browser)
    search_p(browser)
    # search_windows = browser.current_window_handle  # 搜索窗口
    # all_handles = browser.window_handles
    # for handle in all_handles:
    #     if handle != search_windows:
    browser.switch_to_window(browser.window_handles[1])
    sleep(5)
    top = browser.find_element_by_id('J_TabBarBox')    # 累计评论点击
    browser.execute_script("arguments[0].scrollIntoView()", top)   # 累计评论点击
    sleep(4)
    browser.find_element_by_xpath('//*[@id="J_TabBar"]/li[2]/a').click()  # 累计评论点击
    sleep(10)  # username = browser.find_elements_by_xpath('//div[@class="rate-user-info"]/text()')
    get_comment(browser.page_source)  # 必须是局部变量
    print('-------第1页爬取完成-------')

    for i in range(2, 120):
        try:
            # 检测是否有验证码
            browser.find_element_by_xpath('//div[contains(@role,"dialog") and contains(@aria-hidden,"false")]')
            print('-------出现验证码-------')
        except:
            print('------无验证码，安全-------')

        try:
            turn_page(browser)
            get_comment(browser.page_source)
            print('-------第%s页爬取完成-------' % i)
        except:
            print('没有更多评论了,总共抓取评论%s条' % len(data))
            # DataFrame是Python中Pandas库中的一种数据结构，它类似excel，是一种二维表。
        df = pd.DataFrame(data)
        df.to_csv('./taobao_pinglun.csv', index=None, encoding='utf-8-sig', mode='a')
        browser.quit()


def add_cookies(browser):
    # 删除本地登陆存储到本地的cookies
    browser.delete_all_cookies()
    # 读取本地的cookies
    with open('cookies_taobao.json', 'r', encoding='utf-8') as f:
        ListCookies = json.loads(f.read())
    for cookie in ListCookies:
        browser.add_cookie({
            'domain': '.taobao.com',
            'name': cookie['name'],
            'value': cookie['value'],
            'path': '/',
            'expires': None
        })
    browser.get("http://www.taobao.com")
    sleep(4)


# 开始爬取想要的物品,并且提取销量最高的评论
def search_p(browser):
    p = input('你要爬取的物品')
    sleep(3)
    browser.find_element_by_id('q').send_keys(p, Keys.ENTER)
    sleep(3)
    browser.find_element_by_link_text('销量').click()
    sleep(3)
    shangpings = browser.find_elements_by_xpath('//div[@class="items"]/div[@class="item J_MouserOnverReq  "]/div[1]')
    shangpings[0].click()
    sleep(3)


# 获取评论
def get_comment(text):
    source = etree.HTML(text)
    info_list = source.xpath('//div[@class="rate-grid"]/table/tbody/tr')
    for i in range(len(info_list)):
        item = {}
        # item['用户名'] = username[i].replace('<span>', '').repalce('<span>', '')
        if info_list[i].xpath('./td[1]/div[@class="tm-rate-premiere"]'):  # 如果是有追加评论的话
            item['评论内容'] = info_list[i].xpath('./td[1]/div[@class="tm-rate-premiere"]//div[@class="tm-rate-content"]/div[@class="tm-rate-fulltxt"]/text()')[0]
            item['评论时间'] = info_list[i].xpath('./td[1]/div[@class="tm-rate-premiere"]/div[@class="tm-rate-tag"]//div[@class="tm-rate-date"]/text()')[0]
            item['追加评论'] = info_list[i].xpath('./td[1]/div[@class="tm-rate-append"]//div[@class="tm-rate-content"]/div[@class="tm-rate-fulltxt"]/text()')[0]
        elif info_list[i].xpath('./td[1]/div[@class="tm-rate-content tm-rate-content-collapse"]'):  # 刷单
            item['评论内容'] = '刷单！！！！！！'
        else:
            item['评论内容'] = info_list[i].xpath('./td[1]/div[@class="tm-rate-content"]/div[@class="tm-rate-fulltxt"]/text()')[0]
            item['评论时间'] = info_list[i].xpath('./td[1]/div[@class="tm-rate-date"]/text()')[0]
            item['追加评论'] = ''
        print(item)
        data.append(item)


# 下一页
def turn_page(browser):
    next_page = browser.find_element_by_xpath('//div[@class="rate-page"]/div[@class="rate-paginator"]//a[contains(text(),"下一页>>")]')
    browser.execute_script("arguments[0].click()", next_page)
    sleep(8)


if __name__ == '__main__':
    main()

















